package com.ma.crawl_book.controller;

import com.ma.crawl_book.model.SectionBean;
import com.ma.crawl_book.model.SectionContent;
import com.ma.crawl_book.model.SectionCustom;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

import java.util.ArrayList;
import java.util.List;

@Component
public class CrawlSection implements PageProcessor {

    private Site site = Site.me().setRetryTimes(5).setSleepTime(1000);

    private static List<String> urls = new ArrayList<>();
    private static int mBookId;
    @Autowired
    SectionController controller;

    private static List<SectionCustom> sections = new ArrayList<>();
    @Override
    public void process(Page page) {
        // 三级页面（书籍章节列表页）
        if (page.getUrl().regex("http://www.23us.so/files/article/html/[0-9]*/[0-9]*/index.html").match()) {
            urls = page.getHtml().xpath("//td[@class='L']/a/@href").all();
            page.addTargetRequests(urls);
        }

        // 四级页面（书籍阅读页）
        if (page.getUrl().regex("http://www.23us.so/files/article/html/[0-9]*/[0-9]*/[0-9]*.html").match()) {
            String url = page.getUrl().get();
            SectionCustom sectionCustom = new SectionCustom();
            SectionBean sectionBean = new SectionBean();
            SectionContent sectionContent = new SectionContent();
            for (int i = 0; i < urls.size(); i++) {
                if (url.equals(urls.get(i))){
                    sectionBean.setPosition(i);
                }
            }
            String bookName = page.getHtml().xpath("//*[@id='amain']/dl/dt/a[3]/text()").get();
            // 章节名称
            String title = page.getHtml().xpath("//*[@id='amain']/dl/dd[1]/h1/text()").get();
            // 章节内容
            String contentOld = page.getHtml().xpath("//*[@id='contents']").get();
            String content = contentOld.substring(contentOld.indexOf(">") + 1, contentOld.indexOf("</dd>")).replace("<br>","");
            sectionBean.setBookId(mBookId);
            //添加章节Bean
            sectionCustom.setSectionBean(sectionBean);
            sectionBean.setName(title);
            sectionBean.setBookName(bookName);
            sectionContent.setContent(content);
            //添加章节信息Bean
            sectionCustom.setSectionContent(sectionContent);
            sections.add(sectionCustom);
        }
    }

    @Override
    public Site getSite() {
        return site;
    }


    public void startCrawl(int bookId, String bookSectionUrl) {
        mBookId = bookId;
        sections.clear();
        Spider.create(new CrawlSection()).addUrl(bookSectionUrl).thread(500).run();
        for (SectionCustom section : sections){
            try {
//                System.out.println(sectionBean.getBookName() + "  " + sectionBean.getPosition());
                //插入章节Bean
                int sectionId = controller.insertSection(section.getSectionBean());
                SectionContent sectionContent = section.getSectionContent();
                sectionContent.setId(sectionId);
                controller.insertSectionContent(sectionContent);
            }catch (Exception e){
                System.out.println(section.getSectionBean().toString());
            }
        }
    }
}
